Source code for nlp_architect.pipelines.spacy_bist

# ******************************************************************************
# Copyright 2017-2018 Intel Corporation
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
# ******************************************************************************
from os import path, remove, makedirs

from nlp_architect.common.core_nlp_doc import CoreNLPDoc
from import ConllEntry
from nlp_architect.models.bist_parser import BISTModel
from nlp_architect import LIBRARY_OUT
from import download_unlicensed_file, uncompress_file
from import validate
from nlp_architect.utils.text import SpacyInstance

[docs]class SpacyBISTParser(object): """Main class which handles parsing with Spacy-BIST parser. Args: verbose (bool, optional): Controls output verbosity. spacy_model (str, optional): Spacy model to use (see bist_model (str, optional): Path to a .model file to load. Defaults pre-trained model'. """ dir = LIBRARY_OUT / "bist-pretrained" _pretrained = dir / "bist.model" def __init__(self, verbose=False, spacy_model="en", bist_model=None): validate( (verbose, bool), (spacy_model, str, 0, 1000), (bist_model, (type(None), str), 0, 1000) ) if not bist_model: print("Using pre-trained BIST model.") _download_pretrained_model() bist_model = SpacyBISTParser._pretrained self.verbose = verbose self.bist_parser = BISTModel() self.bist_parser.load(bist_model if bist_model else SpacyBISTParser._pretrained) self.spacy_parser = SpacyInstance(spacy_model, disable=["ner", "vectors", "textcat"]).parser
[docs] def to_conll(self, doc_text): """Converts a document to CoNLL format with spacy POS tags. Args: doc_text (str): raw document text. Yields: list of ConllEntry: The next sentence in the document in CoNLL format. """ validate((doc_text, str)) for sentence in self.spacy_parser(doc_text).sents: sentence_conll = [ ConllEntry( 0, "*root*", "*root*", "ROOT-POS", "ROOT-CPOS", "_", -1, "rroot", "_", "_" ) ] i_tok = 0 for tok in sentence: if self.verbose: print(tok.text + "\t" + tok.tag_) if not tok.is_space: pos = tok.tag_ text = tok.text if text != "-" or pos != "HYPH": pos = _spacy_pos_to_ptb(pos, text) token_conll = ConllEntry( i_tok + 1, text, tok.lemma_, pos, pos, tok.ent_type_, -1, "_", "_", tok.idx, ) sentence_conll.append(token_conll) i_tok += 1 if self.verbose: print("-----------------------\ninput conll form:") for entry in sentence_conll: print(str( + "\t" + entry.form + "\t" + entry.pos + "\t") yield sentence_conll
[docs] def parse(self, doc_text, show_tok=True, show_doc=True): """Parse a raw text document. Args: doc_text (str) show_tok (bool, optional): Specifies whether to include token text in output. show_doc (bool, optional): Specifies whether to include document text in output. Returns: CoreNLPDoc: The annotated document. """ validate((doc_text, str), (show_tok, bool), (show_doc, bool)) doc_conll = self.to_conll(doc_text) parsed_doc = CoreNLPDoc() if show_doc: parsed_doc.doc_text = doc_text for sent_conll in self.bist_parser.predict_conll(doc_conll): parsed_sent = [] conj_governors = {"and": set(), "or": set()} for tok in sent_conll: gov_id = int(tok.pred_parent_id) rel = tok.pred_relation if tok.form != "*root*": if tok.form.lower() == "and": conj_governors["and"].add(gov_id) if tok.form.lower() == "or": conj_governors["or"].add(gov_id) if rel == "conj": if gov_id in conj_governors["and"]: rel += "_and" if gov_id in conj_governors["or"]: rel += "_or" parsed_tok = { "start": tok.misc, "len": len(tok.form), "pos": tok.pos, "ner": tok.feats, "lemma": tok.lemma, "gov": gov_id - 1, "rel": rel, } if show_tok: parsed_tok["text"] = tok.form parsed_sent.append(parsed_tok) if parsed_sent: parsed_doc.sentences.append(parsed_sent) return parsed_doc
def _download_pretrained_model(): """Downloads the pre-trained BIST model if non-existent.""" if not path.isfile(SpacyBISTParser.dir / "bist.model"): print("Downloading pre-trained BIST model...") zip_path = SpacyBISTParser.dir / "" makedirs(SpacyBISTParser.dir, exist_ok=True) download_unlicensed_file( "", "", zip_path, ) print("Unzipping...") uncompress_file(zip_path, outpath=str(SpacyBISTParser.dir)) remove(zip_path) print("Done.") def _spacy_pos_to_ptb(pos, text): """ Converts a Spacy part-of-speech tag to a Penn Treebank part-of-speech tag. Args: pos (str): Spacy POS tag. text (str): The token text. Returns: ptb_tag (str): Standard PTB POS tag. """ validate((pos, str, 0, 30), (text, str, 0, 1000)) ptb_tag = pos if text in ["...", "—"]: ptb_tag = ":" elif text == "*": ptb_tag = "SYM" elif pos == "AFX": ptb_tag = "JJ" elif pos == "ADD": ptb_tag = "NN" elif text != pos and text in [",", ".", ":", "``", "-RRB-", "-LRB-"]: ptb_tag = text elif pos in ["NFP", "HYPH", "XX"]: ptb_tag = "SYM" return ptb_tag